{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/train-ms-en.tar.gz\n",
    "# !tar -zxf train-ms-en.tar.gz\n",
    "# !rm train-ms-en.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/train-en-ms.tar.gz\n",
    "# !tar -zxf train-en-ms.tar.gz\n",
    "# !rm train-en-ms.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "left.txt  right.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "left.txt  right.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls train-en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 3712555/3712555 [00:12<00:00, 289962.37it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'w') as fopen_jsonl:\n",
    "    with open('train/left.txt') as fopen:\n",
    "        left = fopen.read().split('\\n')\n",
    "\n",
    "    with open('train/right.txt') as fopen:\n",
    "        right = fopen.read().split('\\n')\n",
    "    \n",
    "    for i in tqdm(range(len(left))):\n",
    "        if len(left[i]) and len(right[i]) and len(left[i].split()) < 256 and len(right[i].split()) < 256:\n",
    "            d = {\"translation\": {\"src\": left[i], \"tgt\": right[i], 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 3807616/3807616 [00:17<00:00, 214482.05it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('train-en/left.txt') as fopen:\n",
    "        left = fopen.read().split('\\n')\n",
    "\n",
    "    with open('train-en/right.txt') as fopen:\n",
    "        right = fopen.read().split('\\n')\n",
    "    \n",
    "    for i in tqdm(range(len(left))):\n",
    "        if len(left[i]) and len(right[i]) and len(left[i].split()) < 256 and len(right[i].split()) < 256:\n",
    "            d = {\"translation\": {\"src\": left[i], \"tgt\": right[i], 'prefix': 'terjemah Inggeris ke Melayu: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 11786274/11786274 [01:00<00:00, 193570.10it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('eng_Latn-zsm_Latn.left') as fopen:\n",
    "        left = fopen.read().split('\\n')\n",
    "    \n",
    "    with open('eng_Latn-zsm_Latn.right') as fopen:\n",
    "        right = fopen.read().split('\\n')\n",
    "        \n",
    "    for i in tqdm(range(len(left))):\n",
    "        if len(left[i]) and len(right[i]) and len(left[i].split()) < 256 and len(right[i].split()) < 256:\n",
    "            d = {\"translation\": {\"src\": left[i], \"tgt\": right[i], 'prefix': 'terjemah Inggeris ke Melayu: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            d = {\"translation\": {\"src\": right[i], \"tgt\": left[i], 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████| 998/998 [00:00<00:00, 160404.48it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('test.json', 'w') as fopen_jsonl:\n",
    "    with open('eng_Latn.dev') as fopen:\n",
    "        left = fopen.read().split('\\n')\n",
    "    \n",
    "    with open('zsm_Latn.dev') as fopen:\n",
    "        right = fopen.read().split('\\n')\n",
    "        \n",
    "    for i in tqdm(range(len(left))):\n",
    "        if len(left[i]) and len(right[i]) and len(left[i].split()) < 256 and len(right[i].split()) < 256:\n",
    "            d = {\"translation\": {\"src\": left[i], \"tgt\": right[i], 'prefix': 'terjemah Inggeris ke Melayu: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            d = {\"translation\": {\"src\": right[i], \"tgt\": left[i], 'prefix': 'terjemah Melayu ke Inggeris: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43, 39)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(left[0].split()), len(right[0].split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -n 1000 train.json > test.json"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
